#!/usr/bin/env bash
# Copyright (c) 2018 Presto Labs Pte. Ltd.
# Author: chensili

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(git rev-parse --show-toplevel)"
LOG_DIR="${REPO_ROOT}/../log"

DATETIME="$(date +%Y%m%d-%H%M%S%z)"
LOG_FILE="log_download_s3.${DATETIME}"

pkill -f "aws s3 sync"

mkdir -p "${LOG_DIR}"
cd "${REPO_ROOT}"

source "${SCRIPT_DIR}/py_bootstrap.sh"

BUCKET="luke.corp.prestolabs.io"
if [[ ${#} -eq 0 ]]; then
  SRC_DIR="s3://${BUCKET}/data/flow"
  DST_DIR="/remote/iosg/coin/data/flow"
else
  SRC_DIR="s3://${BUCKET}/data/flow$1"
  DST_DIR="/remote/iosg/coin/data/flow$1"
fi

# https://docs.aws.amazon.com/cli/latest/topic/s3-config.html
aws configure set default.s3.max_bandwidth 5MB/s

# https://github.com/aws/aws-cli/pull/2747
# Ignore old data.
OPTIONS="--no-progress --exclude *20180* --exclude *gdax*"

echo "BUCKET='${BUCKET}'"
date
aws s3 sync ${OPTIONS} "${SRC_DIR}" "${DST_DIR}" \
  2>> "${LOG_DIR}/${LOG_FILE}"
date
